print "hello"
print "Begins..."

stopList= []
stopList.append(52693)
stopList.append(53140)
stopList.append(54363)
stopList.append(54392)
stopList.append(55828)
stopList.append(55934)
stopList.append(57467)
stopList.append(57570)
stopList.append(58338)
stopList.append(58412)
stopList.append(60033)
stopList.append(60117)
stopList.append(61478)
stopList.append(61968)
stopList.append(62120)
stopList.append(62472)
stopList.append(63641)
stopList.append(64687)
stopList.append(64704)
stopList.append(64723)
stopList.append(64741)
stopList.append(65222)
stopList.append(65504)
stopList.append(67326)
stopList.append(67550)
stopList.append(67961)
stopList.append(69028)
stopList.append(69401)
stopList.append(69552)
stopList.append(70285)
stopList.append(70707)
stopList.append(70815)
print "len(stopList):",len(stopList)

# 2 arguments I can vary
# num of queries
# num of results taken into consideration for each query
NUM_OF_QUERIES_NEEDED_TO_BE_PROCESSED = 10
NUM_OF_RESULTS_TAKEN_INTO_CONSIDERATION = 100

# data structures for storing query view
overallQueryTermDocsDict = {}

# the input file for the machine pangolin:
# option1:
# inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/head_10K_ANDSemanticsTOP10ResultsForQueryView_Part_of"

# option2:
# inputFileName = "/home/diaosi/web-search-engine-wei/polyIRIndexer/head_10K_ANDSemanticsTOP10ResultsForQueryView"

# option3:
# 95K queries, top1000 for each query, a lot of information
inputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/raw_TOP1000_results_for_head_95K_queries_output_from_IRTK"

inputFileHandler = open(inputFileName,"r")

currentLine = inputFileHandler.readline()
nextLine = ""
currentQID = ""

numOfResultsForTheAllQueries = 0
numOfQueriesHavingQID = 0
numOfQueriesHavingSearchContent = 0
numOfQueriesHavingSearchResults = 0


current_num_of_queries_processed = 0

parseFlag = False
while currentLine:
    # sampling parsing line:
    # qid: 701
    if currentLine.startswith("qid:"):
        
        current_num_of_queries_processed += 1
        print "current_num_of_queries_processed:",current_num_of_queries_processed
        if current_num_of_queries_processed > NUM_OF_QUERIES_NEEDED_TO_BE_PROCESSED:
            break
        
        currentQID = currentLine.strip().split(":")[1].strip()
        print "currentQID:",currentQID
        numOfQueriesHavingQID += 1
        if int(currentQID) not in stopList:
            parseFlag = True
        else:
            parseFlag = False
            print "----->",currentQID,"OUT."
    # sample parsing line:
    # Search: u s  oil industry history
    if currentLine.startswith("Search:") and parseFlag:
        elementList = currentLine.strip().split(" ")
        
        currentSearchContent = ""
        
        queryTermList = []
        for element in elementList[1:]:
            if element.strip() != "":
                queryTermList.append( element.strip() )        
        currentQueryLength = len(queryTermList)
        print "currentQueryLength:",currentQueryLength

        for element in queryTermList:
            currentSearchContent += element + " "
        
        print currentSearchContent
        numOfQueriesHavingSearchContent += 1
        
        
        currentQueryTermIndexDict = {}
        
        nextLine = inputFileHandler.readline()
        #print nextLine.strip()
        if nextLine.strip() != "":
            # skip one more line for the following line format:
            # [2013-06-01 21:02:53] 38871 terms with their lists length have been loaded. 
            nextLine = inputFileHandler.readline()
            
            nextLineElements = nextLine.strip().split(" ")
            # print "nextLineElements:",nextLineElements
            # sample parsing line: 
            # oil:0 industry:1 history:2 u:3 s:4
            for element in nextLineElements:
                if element.split(":")[0] not in currentQueryTermIndexDict:
                    currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
            
            print "currentQueryTermIndexDict:",currentQueryTermIndexDict
            # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
            
            for indexNumber in currentQueryTermIndexDict:
                if currentQueryTermIndexDict[indexNumber] not in overallQueryTermDocsDict:
                    overallQueryTermDocsDict[ currentQueryTermIndexDict[indexNumber] ] = {}
                else:
                    # because the query term has already in the dict, so there is no need to add this key and the empty dict {} again.
                    pass
            
            
            
            # Now, it is time to read the threshold line
            currentLine = inputFileHandler.readline()
            if len( currentLine.strip().split(":") ) == 2:
                print currentLine.strip().split(":")[0],currentLine.strip().split(":")[1]
            else:
                print "format problem"
                exit()
            
            # Now, it is time to read the computation method line
            currentLine = inputFileHandler.readline()
            if len( currentLine.strip().split(":") ) == 2:
                print currentLine.strip().split(":")[0],currentLine.strip().split(":")[1]
            else:
                print "format problem"
                exit()
                        
            # Now, it is time to construct the training examples
            currentLine = inputFileHandler.readline()
            numOfResultsForTheCurrentQuery = 0
            currentResultLines = []
            # print "currentLine:",currentLine
            print "currentLine:",currentLine
            lineElements = currentLine.strip().split(" ")
            while len(lineElements) == 35:
                numOfResultsForTheCurrentQuery += 1
                
                # important test part for me
                if numOfResultsForTheCurrentQuery > NUM_OF_RESULTS_TAKEN_INTO_CONSIDERATION:
                    break

                currentResultLines.append(currentLine.strip())
                
                # here, should extract the useful info from this current line
                for indexNumber in currentQueryTermIndexDict:
                    # This is currently ONLY for AND semantics
                    # print "lineElements[34]:",lineElements[34]
                    # print "overallQueryTermDocsDict[indexNumber]:",overallQueryTermDocsDict[indexNumber]
                    rank_in_int_format = int(lineElements[0])
                    sub_key = lineElements[34] + "_" + lineElements[33]
                    if sub_key not in overallQueryTermDocsDict[ currentQueryTermIndexDict[indexNumber] ]:
                        overallQueryTermDocsDict[ currentQueryTermIndexDict[indexNumber] ][ sub_key ] = []
                        overallQueryTermDocsDict[ currentQueryTermIndexDict[indexNumber] ][ sub_key ].append(rank_in_int_format)
                    else:
                        overallQueryTermDocsDict[ currentQueryTermIndexDict[indexNumber] ][ sub_key ].append(rank_in_int_format)
                    # The OR semantics which I have NOT handled.
                    # Need to handle this as well.
                
                
                currentLine = inputFileHandler.readline()
                lineElements = currentLine.strip().split(" ")
               
            numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
            
            # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
            # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
            # This sampling method has been developed on 2013/01/09
            
            print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
            print "len(currentResultLines):",len(currentResultLines)
            # print "len(queryTermList):",len(queryTermList)
            
            if len(queryTermList) > 10:
                # The result list returned only have 10 holds so it only support the query which has the maximun length of 10
                pass
            else:
                if numOfResultsForTheCurrentQuery == 0:
                    pass
                    # print "currentQID:",currentQID
                    # print "currentSearchContent:",currentSearchContent
                    # print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
                    # print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                else:
                    numOfQueriesHavingSearchResults += 1
            print
    
    currentLine = inputFileHandler.readline()

# checkpoint 1
#for queryTerm in overallQueryTermDocsDict:
#    print "queryTerm:",queryTerm,"overallQueryTermDocsDict:",overallQueryTermDocsDict[queryTerm]
    
# checkpoint 2
#print "term: representative"
#for currentDocIDAndTrecIDSubkey in overallQueryTermDocsDict["representative"]:
#    print currentDocIDAndTrecIDSubkey,overallQueryTermDocsDict["representative"][currentDocIDAndTrecIDSubkey]

inputFileHandler.close()


outputFileName = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/head_10_ANDSemantics_TOP100_QueryView"
outputFileHandler = open(outputFileName,"w")

# double loop
for term in overallQueryTermDocsDict:
    outputFileHandler.write(term + " " + str( len(overallQueryTermDocsDict[term]) ) + "\n")
    print term,str( len(overallQueryTermDocsDict[term]) )
    
    '''
    # for debug ONLY
    for key in overallQueryTermDocsDict[term]:
        if len( overallQueryTermDocsDict[term][key] ) != 1:
            print term,key,overallQueryTermDocsDict[term][key]
    '''
    
    # print term,overallQueryTermDocsDict[term]
    
    docIDAndTrecIDSubkeyList = overallQueryTermDocsDict[term].keys()
    docIDAndTrecIDSubkeyList.sort()
    
    for currentDocIDAndTrecIDSubkey in docIDAndTrecIDSubkeyList:
        outputFileHandler.write(currentDocIDAndTrecIDSubkey + " ")
        outputFileHandler.write( str( len( overallQueryTermDocsDict[term][currentDocIDAndTrecIDSubkey] ) ) + " ")
        for theRank in overallQueryTermDocsDict[term][currentDocIDAndTrecIDSubkey]:
            outputFileHandler.write(str(theRank) + " ")
        outputFileHandler.write("\n")
        # print currentDocIDAndTrecIDSubkey,overallQueryTermDocsDict["representative"][currentDocIDAndTrecIDSubkey]
    outputFileHandler.write("\n")

print "numOfQueriesHavingQID:",numOfQueriesHavingQID
print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
print "numOfQueriesHavingSearchResults:",numOfQueriesHavingSearchResults
outputFileHandler.close()

print "Ends."

